package exp;

import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import tools.FileReaderEnhance;
import tools.ObjectTools;
import tools.webkit.Webpage;
import exp.web.cnn.CNNExtraction;
import exp.web.usatoday.USATodayExtraction;

/**
 * Extract highlights for each topic in CNN / USATODAY
 * <br>
 * for websites other than CNN / USATODAY, extract only title
 * 
 * @author Lanjun
 * 
 */
public class EXP5 {
	static final Pattern TITLE = Pattern.compile("<title>(.*?)</title>");

	public void run(String [] topicList, String inputPath, String outputPath) {
		for (String topic : topicList) {
			File outputFile = new File(outputPath + topic);
			if (outputFile.exists()) {
				System.out.println("EXP5: " + outputFile.getAbsolutePath() + " already processed!");
				continue;
			}
			HashMap<String, URLContent> done = new HashMap<>();
			for (File file : (new File(inputPath)).listFiles()) {
				if (!file.getName().startsWith(topic)) {
					continue;
				}
				@SuppressWarnings("unchecked")
				List<Webpage> w = (ArrayList<Webpage>)ObjectTools.readFromFile(file);
				for (Webpage page : w) {
					try {
						URL url = new URL(page.realURL);
						String host = url.getHost();
						if (host.equals("edition.cnn.com") 
								&& (!page.realURL.startsWith("http://edition.cnn.com/video"))
								&& (!page.realURL.startsWith("http://edition.cnn.com/SPECIALS"))
								&& (!page.realURL.startsWith("http://edition.cnn.com/interactive"))) {
							//unify the URL before processing
							page.realURL = CNNExtraction.unify(page.realURL);
							if (done.containsKey(page.realURL)) {
//								System.out.println("Existed:" + page.realURL);
								URLContent hc = done.get(page.realURL);
								if (!hc.shortURL.contains(page.url)) {
									hc.shortURL.add(page.url);
								}
								done.put(page.realURL, hc);
							} else {
//								System.out.println("New:" + page.realURL);
								URLContent hc = CNNExtraction.getCNN(page.content, page.realURL);
								hc.shortURL = new ArrayList<>();
								hc.shortURL.add(page.url);
//								System.out.println(hc.highlights);
								done.put(page.realURL, hc);
							}
						} else if (host.equals("www.usatoday.com")
								&& (!page.realURL.startsWith("http://www.usatoday.com/videos"))) {
							page.realURL = USATodayExtraction.unify(page.realURL);
							if (done.containsKey(page.realURL)) {
//								System.out.println("Existed:" + page.realURL);
								URLContent hc = done.get(page.realURL);
								if (!hc.shortURL.contains(page.url)) {
									hc.shortURL.add(page.url);
								}
								done.put(page.realURL, hc);
							} else {
//								System.out.println(page.realURL);
								URLContent hc = USATodayExtraction.getUSAToday(page.content, page.realURL);
								hc.shortURL = new ArrayList<>();
								hc.shortURL.add(page.url);
//								System.out.println(hc.highlights);
								done.put(page.realURL, hc);
							}
						} else {
							//only title will be extracted
							if (!done.containsKey(page.realURL)) {
								URLContent hc = new URLContent();
								hc.url = page.realURL;
								hc.shortURL = new ArrayList<>();
								hc.shortURL.add(page.url);
								if (page.content == null) {
									done.put(page.realURL, hc);
								} else {
									//get title
									Matcher tm = TITLE.matcher(page.content);
									while (tm.find()) {
										String title = tm.group(1);
										hc.title = title;
									}
									done.put(page.realURL, hc);
								}
							} else {
								URLContent hc = done.get(page.realURL);
								if (!hc.shortURL.contains(page.url)) {
									hc.shortURL.add(page.url);
								}
								done.put(page.realURL, hc);
							}
						}
					} catch (MalformedURLException e) {
						e.printStackTrace();
					}
				}
				w.clear();
				w = null;
				System.gc();
			}
			ObjectTools.writeToFile(done, outputFile);
			done.clear();
			done = null;
			System.gc();
		}
	}
	
	public static void main(String[] args) {
//		try {
//			System.setOut(new PrintStream(new File("./logs/EXP5.out.log")));
//		} catch (FileNotFoundException e) {
//			e.printStackTrace();
//		}
		
		String[] topicList = FileReaderEnhance.readToString("data/_newData/topiclist", "UTF-8").split("\n");
		String inputPath = "data/_newData/obj/urlContent2/";
		String outputPath = "data/_newData/obj/highlights/";
		
		EXP5 exp5 = new EXP5();
		exp5.run(topicList, inputPath, outputPath);
		
	}
}
